import os
import time
from multiprocessing import Pool 


tree = "./menu_tree_to_dict_embedding.json"
input_files = os.listdir("./08.split_file")
input_dirs = [os.path.join("./08.split_file/", x ) for x in input_files]

output_dir = "./09.cluster_result"
os.makedirs(output_dir, exist_ok=True)


def exec_cluster(input,tree,output):
    cmd = f"python ./09.label_concept_tree_cluster.py -label_df {input} -concept_json {tree} -output_file {output}"
    print(cmd)
    os.system(cmd)
    


tasks = []
for inputdir in input_dirs:
    inputs = os.listdir(inputdir)
    for file in os.listdir(inputdir):
        input_file_path = os.path.join(inputdir, file)
        print(input_file_path)
        out_dir = os.path.join("./09.cluster_result", inputdir.split("/")[-1])
        os.makedirs(out_dir, exist_ok=True)
        output_file_path = os.path.join(out_dir, file.split(".")[0] + ".json")
        print(output_file_path)
        if not os.path.exists(output_file_path):
            tasks.append((input_file_path, tree, output_file_path))


# for i in tasks:
#     exec_cluster(i[0], i[1], i[2])

with Pool(250) as pool:
    pool.starmap(exec_cluster, tasks)